{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "texts = ['orange banana apple grape', 'banana apple apple ', 'grape', 'orange apple']\n",
    "\n",
    "cv = CountVectorizer()\n",
    "cv_fit = cv.fit_transform(texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'orange': 3, 'banana': 1, 'apple': 0, 'grape': 2}\n"
     ]
    }
   ],
   "source": [
    "\n",
    "print(cv.vocabulary_) # 按照字母顺序"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "matrix([[4, 2, 2, 2]], dtype=int64)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum_words = cv_fit.sum(axis=0)\n",
    "sum_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('orange', 2), ('banana', 2), ('apple', 4), ('grape', 2)]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words_freq = [(word, sum_words[0, idx]) for word, idx in cv.vocabulary_.items()]\n",
    "words_freq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('apple', 4), ('orange', 2), ('banana', 2), ('grape', 2)]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 按照词频从大到小排序\n",
    "words_freq =sorted(words_freq, key = lambda x: x[1], reverse=True)\n",
    "words_freq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  (0, 3)\t1\n",
      "  (0, 1)\t1\n",
      "  (0, 0)\t1\n",
      "  (0, 2)\t1\n",
      "  (1, 1)\t1\n",
      "  (1, 0)\t2\n",
      "  (2, 2)\t1\n",
      "  (3, 3)\t1\n",
      "  (3, 0)\t1\n"
     ]
    }
   ],
   "source": [
    "print(cv_fit) # 第一个字符串 顺序为3的词语 出现次数为1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1 1 1 1]\n",
      " [2 1 0 0]\n",
      " [0 0 1 0]\n",
      " [1 0 0 1]]\n"
     ]
    }
   ],
   "source": [
    "print(cv_fit.toarray())  # 第一个字符串，排名0,1,2,3词汇（apple，banana，grape，orange）出现的频率都为1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "TFIDF\n",
    "词频 TF  ： 某个词在文章中出现的次数/文章的总词数\n",
    "\n",
    "你文档频率 IDF = log(总样本数/<包含该词的文档数 + 1>)\n",
    "\n",
    "TFIDF=TF * IDF\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'orange': 3, 'banana': 1, 'apple': 0, 'grape': 2}\n",
      "  (0, 2)\t0.5230350301866413\n",
      "  (0, 0)\t0.423441934145613\n",
      "  (0, 1)\t0.5230350301866413\n",
      "  (0, 3)\t0.5230350301866413\n",
      "  (1, 0)\t0.8508160982744233\n",
      "  (1, 1)\t0.5254635733493682\n",
      "  (2, 2)\t1.0\n",
      "  (3, 0)\t0.6292275146695526\n",
      "  (3, 3)\t0.7772211620785797\n",
      "[[0.42344193 0.52303503 0.52303503 0.52303503]\n",
      " [0.8508161  0.52546357 0.         0.        ]\n",
      " [0.         0.         1.         0.        ]\n",
      " [0.62922751 0.         0.         0.77722116]]\n"
     ]
    }
   ],
   "source": [
    "idf = TfidfVectorizer()\n",
    "idf_fit = idf.fit_transform(texts)\n",
    "print(idf.vocabulary_)\n",
    "print(idf_fit) # (0, 2)\t0.5230350301866413  第一个字符串 中第一个grape词， tfidf=tf * idf = 2 / 4 * log(4 / 3) = \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.42344193 0.52303503 0.52303503 0.52303503]\n",
      " [0.8508161  0.52546357 0.         0.        ]\n",
      " [0.         0.         1.         0.        ]\n",
      " [0.62922751 0.         0.         0.77722116]]\n"
     ]
    }
   ],
   "source": [
    "print(idf_fit.toarray())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('orange', 3)\n",
      "('banana', 1)\n",
      "('apple', 0)\n",
      "('grape', 2)\n"
     ]
    }
   ],
   "source": [
    "for item in cv.vocabulary_.items():\n",
    "    print(item)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
